{
 "cells": [
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "#### VPG的算法思想\n",
    "\n",
    "VPG算法可用于离散动作或连续动作，核心思想是利用神经网络拟合策略$\\pi(a|s)$，损失函数为：\n",
    "\n",
    "离散动作：\n",
    "$$a_{t}=max \\pi(A|s_{t};\\theta) \\\\ Loss(\\theta)=Adv(s_{t},a_{t})*log\\pi(a_{t}|s_{t};\\theta)$$\n",
    "\n",
    "连续动作：\n",
    "$$\\pi(a_{t}|s_{t};\\theta^{\\mu,\\sigma})=N(\\mu(s_{t}),\\sigma(s_{t})) \\\\ \n",
    "a_{t}\\sim \\pi(A|s_{t};\\theta^{\\mu,\\sigma}) \\\\\n",
    "Loss(\\theta^{\\mu,\\sigma})=Adv(s_{t},a_{t})*log\\pi(a_{t}|s_{t};\\theta^{\\mu,\\sigma})$$\n",
    "\n",
    "vpg_core.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import scipy.signal\n",
    "from gym.spaces import Box, Discrete\n",
    "\n",
    "import torch\n",
    "import torch.nn as nn\n",
    "from torch.distributions.normal import Normal\n",
    "from torch.distributions.categorical import Categorical\n",
    "\n",
    "\n",
    "def combined_shape(length, shape=None):\n",
    "    if shape is None:\n",
    "        return (length,)\n",
    "    return (length, shape) if np.isscalar(shape) else (length, *shape)\n",
    "\n",
    "\n",
    "def mlp(sizes, activation, output_activation=nn.Identity):\n",
    "    layers = []\n",
    "    for j in range(len(sizes)-1):\n",
    "        act = activation if j < len(sizes)-2 else output_activation\n",
    "        layers += [nn.Linear(sizes[j], sizes[j+1]), act()]\n",
    "    return nn.Sequential(*layers)\n",
    "\n",
    "\n",
    "def count_vars(module):\n",
    "    return sum([np.prod(p.shape) for p in module.parameters()])\n",
    "\n",
    "\n",
    "def discount_cumsum(x, discount):\n",
    "    \"\"\"\n",
    "    magic from rllab for computing discounted cumulative sums of vectors.\n",
    "\n",
    "    input: \n",
    "        vector x, \n",
    "        [x0, \n",
    "         x1, \n",
    "         x2]\n",
    "\n",
    "    output:\n",
    "        [x0 + discount * x1 + discount^2 * x2,  \n",
    "         x1 + discount * x2,\n",
    "         x2]\n",
    "    \"\"\"\n",
    "    return scipy.signal.lfilter([1], [1, float(-discount)], x[::-1], axis=0)[::-1]\n",
    "\n",
    "\n",
    "class Actor(nn.Module):\n",
    "\n",
    "    def _distribution(self, obs):\n",
    "        raise NotImplementedError\n",
    "\n",
    "    def _log_prob_from_distribution(self, pi, act):\n",
    "        raise NotImplementedError\n",
    "\n",
    "    def forward(self, obs, act=None):\n",
    "        # Produce action distributions for given observations, and \n",
    "        # optionally compute the log likelihood of given actions under\n",
    "        # those distributions.\n",
    "        pi = self._distribution(obs)\n",
    "        logp_a = None\n",
    "        if act is not None:\n",
    "            logp_a = self._log_prob_from_distribution(pi, act)\n",
    "        return pi, logp_a\n",
    "\n",
    "\n",
    "class MLPCategoricalActor(Actor):\n",
    "    \n",
    "    def __init__(self, obs_dim, act_dim, hidden_sizes, activation):\n",
    "        super().__init__()\n",
    "        self.logits_net = mlp([obs_dim] + list(hidden_sizes) + [act_dim], activation)\n",
    "\n",
    "    def _distribution(self, obs):\n",
    "        logits = self.logits_net(obs)\n",
    "        return Categorical(logits=logits)\n",
    "\n",
    "    def _log_prob_from_distribution(self, pi, act):\n",
    "        return pi.log_prob(act)\n",
    "\n",
    "\n",
    "class MLPGaussianActor(Actor):\n",
    "\n",
    "    def __init__(self, obs_dim, act_dim, hidden_sizes, activation):\n",
    "        super().__init__()\n",
    "        log_std = -0.5 * np.ones(act_dim, dtype=np.float32)\n",
    "        self.log_std = torch.nn.Parameter(torch.as_tensor(log_std))\n",
    "        self.mu_net = mlp([obs_dim] + list(hidden_sizes) + [act_dim], activation)\n",
    "\n",
    "    def _distribution(self, obs):\n",
    "        mu = self.mu_net(obs)\n",
    "        std = torch.exp(self.log_std)\n",
    "        return Normal(mu, std)\n",
    "\n",
    "    def _log_prob_from_distribution(self, pi, act):\n",
    "        return pi.log_prob(act).sum(axis=-1)    # Last axis sum needed for Torch Normal distribution\n",
    "\n",
    "\n",
    "class MLPCritic(nn.Module):\n",
    "\n",
    "    def __init__(self, obs_dim, hidden_sizes, activation):\n",
    "        super().__init__()\n",
    "        self.v_net = mlp([obs_dim] + list(hidden_sizes) + [1], activation)\n",
    "\n",
    "    def forward(self, obs):\n",
    "        return torch.squeeze(self.v_net(obs), -1) # Critical to ensure v has right shape.\n",
    "\n",
    "class MLPActorCritic(nn.Module):\n",
    "    def __init__(self, observation_space, action_space, \n",
    "                 hidden_sizes=(64,64), activation=nn.Tanh):\n",
    "        super().__init__()\n",
    "\n",
    "        obs_dim = observation_space.shape[0]\n",
    "\n",
    "        # policy builder depends on action space\n",
    "        if isinstance(action_space, Box):\n",
    "            self.pi = MLPGaussianActor(obs_dim, action_space.shape[0], hidden_sizes, activation)\n",
    "        elif isinstance(action_space, Discrete):\n",
    "            self.pi = MLPCategoricalActor(obs_dim, action_space.n, hidden_sizes, activation)\n",
    "\n",
    "        # build value function\n",
    "        self.v  = MLPCritic(obs_dim, hidden_sizes, activation)\n",
    "\n",
    "    def step(self, obs):\n",
    "        with torch.no_grad():\n",
    "            pi = self.pi._distribution(obs) # continuous action space: N(mu(s;theta),std)\n",
    "            a = pi.sample()\n",
    "            logp_a = self.pi._log_prob_from_distribution(pi, a)\n",
    "            v = self.v(obs)\n",
    "        return a.numpy(), v.numpy(), logp_a.numpy()\n",
    "\n",
    "    def act(self, obs):\n",
    "        return self.step(obs)[0]"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "vpg.py"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "import numpy as np\n",
    "import torch\n",
    "from torch.optim import Adam\n",
    "import gym\n",
    "import time\n",
    "import spinup.algos.pytorch.vpg.core as core\n",
    "from spinup.utils.logx import EpochLogger\n",
    "from spinup.utils.mpi_pytorch import setup_pytorch_for_mpi, sync_params, mpi_avg_grads\n",
    "from spinup.utils.mpi_tools import mpi_fork, mpi_avg, proc_id, mpi_statistics_scalar, num_procs\n",
    "\n",
    "\n",
    "class VPGBuffer:\n",
    "    \"\"\"\n",
    "    A buffer for storing trajectories experienced by a VPG agent interacting\n",
    "    with the environment, and using Generalized Advantage Estimation (GAE-Lambda)\n",
    "    for calculating the advantages of state-action pairs.\n",
    "    \"\"\"\n",
    "\n",
    "    def __init__(self, obs_dim, act_dim, size, gamma=0.99, lam=0.95):\n",
    "        self.obs_buf = np.zeros(core.combined_shape(size, obs_dim), dtype=np.float32)\n",
    "        self.act_buf = np.zeros(core.combined_shape(size, act_dim), dtype=np.float32)\n",
    "        self.adv_buf = np.zeros(size, dtype=np.float32)\n",
    "        self.rew_buf = np.zeros(size, dtype=np.float32)\n",
    "        self.ret_buf = np.zeros(size, dtype=np.float32)\n",
    "        self.val_buf = np.zeros(size, dtype=np.float32)\n",
    "        self.logp_buf = np.zeros(size, dtype=np.float32)\n",
    "        self.gamma, self.lam = gamma, lam\n",
    "        self.ptr, self.path_start_idx, self.max_size = 0, 0, size\n",
    "\n",
    "    def store(self, obs, act, rew, val, logp):\n",
    "        \"\"\"\n",
    "        Append one timestep of agent-environment interaction to the buffer.\n",
    "        \"\"\"\n",
    "        assert self.ptr < self.max_size     # buffer has to have room so you can store\n",
    "        self.obs_buf[self.ptr] = obs\n",
    "        self.act_buf[self.ptr] = act\n",
    "        self.rew_buf[self.ptr] = rew\n",
    "        self.val_buf[self.ptr] = val\n",
    "        self.logp_buf[self.ptr] = logp\n",
    "        self.ptr += 1\n",
    "\n",
    "    def finish_path(self, last_val=0):\n",
    "        \"\"\"\n",
    "        Call this at the end of a trajectory, or when one gets cut off\n",
    "        by an epoch ending. This looks back in the buffer to where the\n",
    "        trajectory started, and uses rewards and value estimates from\n",
    "        the whole trajectory to compute advantage estimates with GAE-Lambda,\n",
    "        as well as compute the rewards-to-go for each state, to use as\n",
    "        the targets for the value function.\n",
    "\n",
    "        The \"last_val\" argument should be 0 if the trajectory ended\n",
    "        because the agent reached a terminal state (died), and otherwise\n",
    "        should be V(s_T), the value function estimated for the last state.\n",
    "        This allows us to bootstrap the reward-to-go calculation to account\n",
    "        for timesteps beyond the arbitrary episode horizon (or epoch cutoff).\n",
    "        \"\"\"\n",
    "\n",
    "        path_slice = slice(self.path_start_idx, self.ptr)\n",
    "        rews = np.append(self.rew_buf[path_slice], last_val)\n",
    "        vals = np.append(self.val_buf[path_slice], last_val)\n",
    "        \n",
    "        # the next two lines implement GAE-Lambda advantage calculation\n",
    "        deltas = rews[:-1] + self.gamma * vals[1:] - vals[:-1]\n",
    "        self.adv_buf[path_slice] = core.discount_cumsum(deltas, self.gamma * self.lam)\n",
    "        \n",
    "        # the next line computes rewards-to-go, to be targets for the value function\n",
    "        self.ret_buf[path_slice] = core.discount_cumsum(rews, self.gamma)[:-1]\n",
    "        \n",
    "        self.path_start_idx = self.ptr\n",
    "\n",
    "    def get(self):\n",
    "        \"\"\"\n",
    "        Call this at the end of an epoch to get all of the data from\n",
    "        the buffer, with advantages appropriately normalized (shifted to have\n",
    "        mean zero and std one). Also, resets some pointers in the buffer.\n",
    "        \"\"\"\n",
    "        assert self.ptr == self.max_size    # buffer has to be full before you can get\n",
    "        self.ptr, self.path_start_idx = 0, 0\n",
    "        # the next two lines implement the advantage normalization trick\n",
    "        adv_mean, adv_std = mpi_statistics_scalar(self.adv_buf)\n",
    "        self.adv_buf = (self.adv_buf - adv_mean) / adv_std\n",
    "        data = dict(obs=self.obs_buf, act=self.act_buf, ret=self.ret_buf,\n",
    "                    adv=self.adv_buf, logp=self.logp_buf)\n",
    "        return {k: torch.as_tensor(v, dtype=torch.float32) for k,v in data.items()}\n",
    "\n",
    "def vpg(env_fn, actor_critic=core.MLPActorCritic, ac_kwargs=dict(),  seed=0, \n",
    "        steps_per_epoch=4000, epochs=50, gamma=0.99, pi_lr=3e-4,\n",
    "        vf_lr=1e-3, train_v_iters=80, lam=0.97, max_ep_len=1000,\n",
    "        logger_kwargs=dict(), save_freq=10):\n",
    "    \"\"\"\n",
    "    Vanilla Policy Gradient \n",
    "\n",
    "    (with GAE-Lambda for advantage estimation)\n",
    "\n",
    "    Args:\n",
    "        env_fn : A function which creates a copy of the environment.\n",
    "            The environment must satisfy the OpenAI Gym API.\n",
    "\n",
    "        actor_critic: The constructor method for a PyTorch Module with a \n",
    "            ``step`` method, an ``act`` method, a ``pi`` module, and a ``v`` \n",
    "            module. The ``step`` method should accept a batch of observations \n",
    "            and return:\n",
    "\n",
    "            ===========  ================  ======================================\n",
    "            Symbol       Shape             Description\n",
    "            ===========  ================  ======================================\n",
    "            ``a``        (batch, act_dim)  | Numpy array of actions for each \n",
    "                                           | observation.\n",
    "            ``v``        (batch,)          | Numpy array of value estimates\n",
    "                                           | for the provided observations.\n",
    "            ``logp_a``   (batch,)          | Numpy array of log probs for the\n",
    "                                           | actions in ``a``.\n",
    "            ===========  ================  ======================================\n",
    "\n",
    "            The ``act`` method behaves the same as ``step`` but only returns ``a``.\n",
    "\n",
    "            The ``pi`` module's forward call should accept a batch of \n",
    "            observations and optionally a batch of actions, and return:\n",
    "\n",
    "            ===========  ================  ======================================\n",
    "            Symbol       Shape             Description\n",
    "            ===========  ================  ======================================\n",
    "            ``pi``       N/A               | Torch Distribution object, containing\n",
    "                                           | a batch of distributions describing\n",
    "                                           | the policy for the provided observations.\n",
    "            ``logp_a``   (batch,)          | Optional (only returned if batch of\n",
    "                                           | actions is given). Tensor containing \n",
    "                                           | the log probability, according to \n",
    "                                           | the policy, of the provided actions.\n",
    "                                           | If actions not given, will contain\n",
    "                                           | ``None``.\n",
    "            ===========  ================  ======================================\n",
    "\n",
    "            The ``v`` module's forward call should accept a batch of observations\n",
    "            and return:\n",
    "\n",
    "            ===========  ================  ======================================\n",
    "            Symbol       Shape             Description\n",
    "            ===========  ================  ======================================\n",
    "            ``v``        (batch,)          | Tensor containing the value estimates\n",
    "                                           | for the provided observations. (Critical: \n",
    "                                           | make sure to flatten this!)\n",
    "            ===========  ================  ======================================\n",
    "\n",
    "        ac_kwargs (dict): Any kwargs appropriate for the ActorCritic object \n",
    "            you provided to VPG.\n",
    "\n",
    "        seed (int): Seed for random number generators.\n",
    "\n",
    "        steps_per_epoch (int): Number of steps of interaction (state-action pairs) \n",
    "            for the agent and the environment in each epoch.\n",
    "\n",
    "        epochs (int): Number of epochs of interaction (equivalent to\n",
    "            number of policy updates) to perform.\n",
    "\n",
    "        gamma (float): Discount factor. (Always between 0 and 1.)\n",
    "\n",
    "        pi_lr (float): Learning rate for policy optimizer.\n",
    "\n",
    "        vf_lr (float): Learning rate for value function optimizer.\n",
    "\n",
    "        train_v_iters (int): Number of gradient descent steps to take on \n",
    "            value function per epoch.\n",
    "\n",
    "        lam (float): Lambda for GAE-Lambda. (Always between 0 and 1,\n",
    "            close to 1.)\n",
    "\n",
    "        max_ep_len (int): Maximum length of trajectory / episode / rollout.\n",
    "\n",
    "        logger_kwargs (dict): Keyword args for EpochLogger.\n",
    "\n",
    "        save_freq (int): How often (in terms of gap between epochs) to save\n",
    "            the current policy and value function.\n",
    "\n",
    "    \"\"\"\n",
    "\n",
    "    # Special function to avoid certain slowdowns from PyTorch + MPI combo.\n",
    "    setup_pytorch_for_mpi()\n",
    "\n",
    "    # Set up logger and save configuration\n",
    "    logger = EpochLogger(**logger_kwargs)\n",
    "    logger.save_config(locals())\n",
    "\n",
    "    # Random seed\n",
    "    seed += 10000 * proc_id()\n",
    "    torch.manual_seed(seed)\n",
    "    np.random.seed(seed)\n",
    "\n",
    "    # Instantiate environment\n",
    "    env = env_fn()\n",
    "    obs_dim = env.observation_space.shape\n",
    "    act_dim = env.action_space.shape\n",
    "\n",
    "    # Create actor-critic module\n",
    "    ac = actor_critic(env.observation_space, env.action_space, **ac_kwargs)\n",
    "\n",
    "    # Sync params across processes\n",
    "    sync_params(ac)\n",
    "\n",
    "    # Count variables\n",
    "    var_counts = tuple(core.count_vars(module) for module in [ac.pi, ac.v])\n",
    "    logger.log('\\nNumber of parameters: \\t pi: %d, \\t v: %d\\n'%var_counts)\n",
    "\n",
    "    # Set up experience buffer\n",
    "    local_steps_per_epoch = int(steps_per_epoch / num_procs())\n",
    "    buf = VPGBuffer(obs_dim, act_dim, local_steps_per_epoch, gamma, lam)\n",
    "\n",
    "    # Set up function for computing VPG policy loss\n",
    "    def compute_loss_pi(data):\n",
    "        obs, act, adv, logp_old = data['obs'], data['act'], data['adv'], data['logp']\n",
    "\n",
    "        # Policy loss\n",
    "        pi, logp = ac.pi(obs, act)\n",
    "        loss_pi = -(logp * adv).mean()\n",
    "\n",
    "        # Useful extra info\n",
    "        approx_kl = (logp_old - logp).mean().item()\n",
    "        ent = pi.entropy().mean().item()\n",
    "        pi_info = dict(kl=approx_kl, ent=ent)\n",
    "\n",
    "        return loss_pi, pi_info\n",
    "\n",
    "    # Set up function for computing value loss\n",
    "    def compute_loss_v(data):\n",
    "        obs, ret = data['obs'], data['ret']\n",
    "        return ((ac.v(obs) - ret)**2).mean()\n",
    "\n",
    "    # Set up optimizers for policy and value function\n",
    "    pi_optimizer = Adam(ac.pi.parameters(), lr=pi_lr)\n",
    "    vf_optimizer = Adam(ac.v.parameters(), lr=vf_lr)\n",
    "\n",
    "    # Set up model saving\n",
    "    logger.setup_pytorch_saver(ac)\n",
    "\n",
    "    def update():\n",
    "        data = buf.get()\n",
    "\n",
    "        # Get loss and info values before update\n",
    "        pi_l_old, pi_info_old = compute_loss_pi(data)\n",
    "        pi_l_old = pi_l_old.item()\n",
    "        v_l_old = compute_loss_v(data).item()\n",
    "\n",
    "        # Train policy with a single step of gradient descent\n",
    "        pi_optimizer.zero_grad()\n",
    "        loss_pi, pi_info = compute_loss_pi(data)\n",
    "        loss_pi.backward()\n",
    "        mpi_avg_grads(ac.pi)    # average grads across MPI processes\n",
    "        pi_optimizer.step()\n",
    "\n",
    "        # Value function learning\n",
    "        for i in range(train_v_iters):\n",
    "            vf_optimizer.zero_grad()\n",
    "            loss_v = compute_loss_v(data)\n",
    "            loss_v.backward()\n",
    "            mpi_avg_grads(ac.v)    # average grads across MPI processes\n",
    "            vf_optimizer.step()\n",
    "\n",
    "        # Log changes from update\n",
    "        kl, ent = pi_info['kl'], pi_info_old['ent']\n",
    "        logger.store(LossPi=pi_l_old, LossV=v_l_old,\n",
    "                     KL=kl, Entropy=ent,\n",
    "                     DeltaLossPi=(loss_pi.item() - pi_l_old),\n",
    "                     DeltaLossV=(loss_v.item() - v_l_old))\n",
    "\n",
    "    # Prepare for interaction with environment\n",
    "    start_time = time.time()\n",
    "    o, ep_ret, ep_len = env.reset(), 0, 0\n",
    "\n",
    "    # Main loop: collect experience in env and update/log each epoch\n",
    "    for epoch in range(epochs):\n",
    "        for t in range(local_steps_per_epoch):\n",
    "            a, v, logp = ac.step(torch.as_tensor(o, dtype=torch.float32))\n",
    "\n",
    "            next_o, r, d, _ = env.step(a)\n",
    "            ep_ret += r\n",
    "            ep_len += 1\n",
    "\n",
    "            # save and log\n",
    "            buf.store(o, a, r, v, logp)\n",
    "            logger.store(VVals=v)\n",
    "            \n",
    "            # Update obs (critical!)\n",
    "            o = next_o\n",
    "\n",
    "            timeout = ep_len == max_ep_len\n",
    "            terminal = d or timeout\n",
    "            epoch_ended = t==local_steps_per_epoch-1\n",
    "\n",
    "            if terminal or epoch_ended:\n",
    "                if epoch_ended and not(terminal):\n",
    "                    print('Warning: trajectory cut off by epoch at %d steps.'%ep_len, flush=True)\n",
    "                # if trajectory didn't reach terminal state, bootstrap value target\n",
    "                if timeout or epoch_ended:\n",
    "                    _, v, _ = ac.step(torch.as_tensor(o, dtype=torch.float32))\n",
    "                else:\n",
    "                    v = 0\n",
    "                buf.finish_path(v)\n",
    "                if terminal:\n",
    "                    # only save EpRet / EpLen if trajectory finished\n",
    "                    logger.store(EpRet=ep_ret, EpLen=ep_len)\n",
    "                o, ep_ret, ep_len = env.reset(), 0, 0\n",
    "\n",
    "\n",
    "        # Save model\n",
    "        if (epoch % save_freq == 0) or (epoch == epochs-1):\n",
    "            logger.save_state({'env': env}, None)\n",
    "\n",
    "        # Perform VPG update!\n",
    "        update()\n",
    "\n",
    "        # Log info about epoch\n",
    "        logger.log_tabular('Epoch', epoch)\n",
    "        logger.log_tabular('EpRet', with_min_and_max=True)\n",
    "        logger.log_tabular('EpLen', average_only=True)\n",
    "        logger.log_tabular('VVals', with_min_and_max=True)\n",
    "        logger.log_tabular('TotalEnvInteracts', (epoch+1)*steps_per_epoch)\n",
    "        logger.log_tabular('LossPi', average_only=True)\n",
    "        logger.log_tabular('LossV', average_only=True)\n",
    "        logger.log_tabular('DeltaLossPi', average_only=True)\n",
    "        logger.log_tabular('DeltaLossV', average_only=True)\n",
    "        logger.log_tabular('Entropy', average_only=True)\n",
    "        logger.log_tabular('KL', average_only=True)\n",
    "        logger.log_tabular('Time', time.time()-start_time)\n",
    "        logger.dump_tabular()\n",
    "\n",
    "if __name__ == '__main__':\n",
    "    import argparse\n",
    "    parser = argparse.ArgumentParser()\n",
    "    parser.add_argument('--env', type=str, default='HalfCheetah-v2')\n",
    "    parser.add_argument('--hid', type=int, default=64)\n",
    "    parser.add_argument('--l', type=int, default=2)\n",
    "    parser.add_argument('--gamma', type=float, default=0.99)\n",
    "    parser.add_argument('--seed', '-s', type=int, default=0)\n",
    "    parser.add_argument('--cpu', type=int, default=4)\n",
    "    parser.add_argument('--steps', type=int, default=4000)\n",
    "    parser.add_argument('--epochs', type=int, default=50)\n",
    "    parser.add_argument('--exp_name', type=str, default='vpg')\n",
    "    args = parser.parse_args()\n",
    "\n",
    "    mpi_fork(args.cpu)  # run parallel code with mpi\n",
    "\n",
    "    from spinup.utils.run_utils import setup_logger_kwargs\n",
    "    logger_kwargs = setup_logger_kwargs(args.exp_name, args.seed)\n",
    "\n",
    "    vpg(lambda : gym.make(args.env), actor_critic=core.MLPActorCritic,\n",
    "        ac_kwargs=dict(hidden_sizes=[args.hid]*args.l), gamma=args.gamma, \n",
    "        seed=args.seed, steps_per_epoch=args.steps, epochs=args.epochs,\n",
    "        logger_kwargs=logger_kwargs)"
   ]
  }
 ],
 "metadata": {
  "language_info": {
   "name": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
